{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "({0: 'O',\n  1: 'B-地块编码',\n  2: 'I-地块编码',\n  3: 'B-地块位置',\n  4: 'I-地块位置',\n  5: 'B-出让面积',\n  6: 'I-出让面积',\n  7: 'B-土地用途',\n  8: 'I-土地用途',\n  9: 'B-容积率',\n  10: 'I-容积率',\n  11: 'B-起始价',\n  12: 'I-起始价',\n  13: 'B-成交价',\n  14: 'I-成交价',\n  15: 'B-溢价率',\n  16: 'I-溢价率',\n  17: 'B-成交时间',\n  18: 'I-成交时间',\n  19: 'B-受让人',\n  20: 'I-受让人',\n  21: 'B-城市',\n  22: 'I-城市',\n  23: 'B-触发词',\n  24: 'I-触发词'},\n {'O': 0,\n  'B-地块编码': 1,\n  'I-地块编码': 2,\n  'B-地块位置': 3,\n  'I-地块位置': 4,\n  'B-出让面积': 5,\n  'I-出让面积': 6,\n  'B-土地用途': 7,\n  'I-土地用途': 8,\n  'B-容积率': 9,\n  'I-容积率': 10,\n  'B-起始价': 11,\n  'I-起始价': 12,\n  'B-成交价': 13,\n  'I-成交价': 14,\n  'B-溢价率': 15,\n  'I-溢价率': 16,\n  'B-成交时间': 17,\n  'I-成交时间': 18,\n  'B-受让人': 19,\n  'I-受让人': 20,\n  'B-城市': 21,\n  'I-城市': 22,\n  'B-触发词': 23,\n  'I-触发词': 24})"
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigger_label = '触发词'\n",
    "schemas = ['O', 'B-地块编码','I-地块编码', 'B-地块位置', 'I-地块位置', 'B-出让面积', 'I-出让面积', 'B-土地用途', 'I-土地用途', 'B-容积率', 'I-容积率',\n",
    "           'B-起始价', 'I-起始价', 'B-成交价', 'I-成交价', 'B-溢价率', 'I-溢价率', 'B-成交时间', 'I-成交时间', 'B-受让人', 'I-受让人',\n",
    "           'B-城市', 'I-城市', 'B-触发词', 'I-触发词']\n",
    "id2tag = {idx: tag for idx, tag in enumerate(schemas)}\n",
    "tag2id = {tag: idx for idx, tag in enumerate(schemas)}\n",
    "\n",
    "id2tag,tag2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "# 读取原始训练据并进行标准化处理\n",
    "import json\n",
    "import numpy\n",
    "\n",
    "def processlabel(line):\n",
    "    # 初始化空数组\n",
    "    line = line.strip()\n",
    "    if len(line) == 0:\n",
    "        return\n",
    "    linejson = json.loads(line)\n",
    "    nertag = numpy.zeros(len(linejson[\"data\"]))\n",
    "    tokens = [i for i in linejson[\"data\"]]\n",
    "    for (start, stop, tagstr) in linejson[\"label\"]:\n",
    "        if tagstr == \"NN\":\n",
    "            nertag[start:stop] = 0\n",
    "        else:\n",
    "            starttagstr = \"B-\" + tagstr\n",
    "            intenelstr = \"I-\" + tagstr\n",
    "            nertag[start] = tag2id[starttagstr]\n",
    "            nertag[start + 1:stop] = tag2id[intenelstr]\n",
    "    return {\"id\": linejson[\"id\"], \"tokens\": tokens, \"ner_tags\": nertag}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "852\n",
      "426\n"
     ]
    }
   ],
   "source": [
    "#预处理数据，label转化为规范格式。\n",
    "import pandas as pd\n",
    "f=open(\"valid.json\",'r',encoding='utf8')\n",
    "lines = f.readlines()\n",
    "processed = list(map(processlabel, lines))\n",
    "print(len(processed))\n",
    "processNone = list(filter(None, processed))\n",
    "print(len(processNone))\n",
    "pd.DataFrame(processNone).to_json(\"valid1.json\")\n",
    "df = pd.read_json(\"valid1.json\")\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "outputs": [
    {
     "data": {
      "text/plain": "Dataset({\n    features: ['id', 'tokens', 'ner_tags', '__index_level_0__'],\n    num_rows: 426\n})"
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取自定义数据集\n",
    "from datasets import Dataset\n",
    "land_train =  Dataset.from_pandas(df, split=\"train\")\n",
    "land_train"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "outputs": [
    {
     "data": {
      "text/plain": "Dataset({\n    features: ['id', 'tokens', 'ner_tags'],\n    num_rows: 1\n})"
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "land_train"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "Cannot save file into a non-existent directory: '\\kaggle\\working'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mOSError\u001B[0m                                   Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_5844/2641230387.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[0mdf\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mDataFrame\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mlist\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mfilter\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;32mNone\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mprocessed\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      2\u001B[0m \u001B[1;31m#df.to_csv(\"/kaggle/working/train_processed1.csv\",index=False)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 3\u001B[1;33m \u001B[0mdf\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mto_csv\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m\"/kaggle/working/valid_mid.csv\"\u001B[0m\u001B[1;33m,\u001B[0m\u001B[0mmode\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;34m'w'\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mindex\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mFalse\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      4\u001B[0m \u001B[0mreaddf\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread_csv\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m\"/kaggle/working/valid_mid.csv\"\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      5\u001B[0m \u001B[1;31m#清楚转csv后的强制空格\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\core\\generic.py\u001B[0m in \u001B[0;36mto_csv\u001B[1;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, line_terminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001B[0m\n\u001B[0;32m   3561\u001B[0m         )\n\u001B[0;32m   3562\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 3563\u001B[1;33m         return DataFrameRenderer(formatter).to_csv(\n\u001B[0m\u001B[0;32m   3564\u001B[0m             \u001B[0mpath_or_buf\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   3565\u001B[0m             \u001B[0mline_terminator\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mline_terminator\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\formats\\format.py\u001B[0m in \u001B[0;36mto_csv\u001B[1;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, line_terminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001B[0m\n\u001B[0;32m   1178\u001B[0m             \u001B[0mformatter\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfmt\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1179\u001B[0m         )\n\u001B[1;32m-> 1180\u001B[1;33m         \u001B[0mcsv_formatter\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msave\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1181\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1182\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mcreated_buffer\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\formats\\csvs.py\u001B[0m in \u001B[0;36msave\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    239\u001B[0m         \"\"\"\n\u001B[0;32m    240\u001B[0m         \u001B[1;31m# apply compression and byte/text conversion\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 241\u001B[1;33m         with get_handle(\n\u001B[0m\u001B[0;32m    242\u001B[0m             \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfilepath_or_buffer\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    243\u001B[0m             \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmode\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\common.py\u001B[0m in \u001B[0;36mget_handle\u001B[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001B[0m\n\u001B[0;32m    695\u001B[0m     \u001B[1;31m# Only for write methods\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    696\u001B[0m     \u001B[1;32mif\u001B[0m \u001B[1;34m\"r\"\u001B[0m \u001B[1;32mnot\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mmode\u001B[0m \u001B[1;32mand\u001B[0m \u001B[0mis_path\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 697\u001B[1;33m         \u001B[0mcheck_parent_directory\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mhandle\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    698\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    699\u001B[0m     \u001B[1;32mif\u001B[0m \u001B[0mcompression\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\common.py\u001B[0m in \u001B[0;36mcheck_parent_directory\u001B[1;34m(path)\u001B[0m\n\u001B[0;32m    569\u001B[0m     \u001B[0mparent\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mPath\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mparent\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    570\u001B[0m     \u001B[1;32mif\u001B[0m \u001B[1;32mnot\u001B[0m \u001B[0mparent\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mis_dir\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 571\u001B[1;33m         \u001B[1;32mraise\u001B[0m \u001B[0mOSError\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34mfr\"Cannot save file into a non-existent directory: '{parent}'\"\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    572\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    573\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mOSError\u001B[0m: Cannot save file into a non-existent directory: '\\kaggle\\working'"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(list(filter(None, processed)))\n",
    "#df.to_csv(\"/kaggle/working/train_processed1.csv\",index=False)\n",
    "df.to_csv(\"/kaggle/working/valid_mid.csv\",mode='w', index=False)\n",
    "readdf = pd.read_csv(\"/kaggle/working/valid_mid.csv\")\n",
    "#清楚转csv后的强制空格\n",
    "readdf[\"ner_tags\"] = readdf[\"ner_tags\"].map(lambda line: line.replace(\"\\n\", \"\"))\n",
    "readdf.to_csv(\"/kaggle/working/valid.csv\",mode='w', index=False)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-de941c036fb35688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset csv/default to C:\\Users\\xiaofei\\.cache\\huggingface\\datasets\\csv\\default-de941c036fb35688\\0.0.0\\bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a...\n"
     ]
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/2 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "69294827e02c4382b807445742ae5ceb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "  0%|          | 0/2 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b6d631c4e56146c58ce615007dd5e620"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "AttributeError",
     "evalue": "'TextFileReader' object has no attribute 'f'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mParserError\u001B[0m                               Traceback (most recent call last)",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\datasets\\packaged_modules\\csv\\csv.py\u001B[0m in \u001B[0;36m_generate_tables\u001B[1;34m(self, files)\u001B[0m\n\u001B[0;32m    171\u001B[0m             \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 172\u001B[1;33m                 \u001B[1;32mfor\u001B[0m \u001B[0mbatch_idx\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdf\u001B[0m \u001B[1;32min\u001B[0m \u001B[0menumerate\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mcsv_file_reader\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    173\u001B[0m                     \u001B[0mpa_table\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpa\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mTable\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfrom_pandas\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdf\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mschema\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mschema\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001B[0m in \u001B[0;36m__next__\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m   1186\u001B[0m         \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1187\u001B[1;33m             \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget_chunk\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1188\u001B[0m         \u001B[1;32mexcept\u001B[0m \u001B[0mStopIteration\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001B[0m in \u001B[0;36mget_chunk\u001B[1;34m(self, size)\u001B[0m\n\u001B[0;32m   1279\u001B[0m             \u001B[0msize\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mmin\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0msize\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mnrows\u001B[0m \u001B[1;33m-\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_currow\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1280\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mnrows\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0msize\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1281\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001B[0m in \u001B[0;36mread\u001B[1;34m(self, nrows)\u001B[0m\n\u001B[0;32m   1249\u001B[0m             \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1250\u001B[1;33m                 \u001B[0mindex\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mcolumns\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mcol_dict\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_engine\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mnrows\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1251\u001B[0m             \u001B[1;32mexcept\u001B[0m \u001B[0mException\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\io\\parsers\\c_parser_wrapper.py\u001B[0m in \u001B[0;36mread\u001B[1;34m(self, nrows)\u001B[0m\n\u001B[0;32m    224\u001B[0m             \u001B[1;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mlow_memory\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 225\u001B[1;33m                 \u001B[0mchunks\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_reader\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread_low_memory\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mnrows\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    226\u001B[0m                 \u001B[1;31m# destructive to chunks\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\_libs\\parsers.pyx\u001B[0m in \u001B[0;36mpandas._libs.parsers.TextReader.read_low_memory\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\_libs\\parsers.pyx\u001B[0m in \u001B[0;36mpandas._libs.parsers.TextReader._read_rows\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\_libs\\parsers.pyx\u001B[0m in \u001B[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\pandas\\_libs\\parsers.pyx\u001B[0m in \u001B[0;36mpandas._libs.parsers.raise_parser_error\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;31mParserError\u001B[0m: Error tokenizing data. C error: Expected 86 fields in line 3, saw 130\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001B[1;31mAttributeError\u001B[0m                            Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_5844/2308210647.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[1;31m#读取自定义数据集\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      2\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0mdatasets\u001B[0m \u001B[1;32mimport\u001B[0m \u001B[0mload_dataset\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 3\u001B[1;33m land = load_dataset(\"csv\", data_files = {\n\u001B[0m\u001B[0;32m      4\u001B[0m     \u001B[1;34m'train'\u001B[0m\u001B[1;33m:\u001B[0m \u001B[1;33m[\u001B[0m\u001B[1;34m'train.csv'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      5\u001B[0m     \u001B[1;34m'validation'\u001B[0m \u001B[1;33m:\u001B[0m \u001B[1;33m[\u001B[0m\u001B[1;34m'valid.csv'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\datasets\\load.py\u001B[0m in \u001B[0;36mload_dataset\u001B[1;34m(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, keep_in_memory, save_infos, revision, use_auth_token, task, streaming, script_version, **config_kwargs)\u001B[0m\n\u001B[0;32m   1662\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1663\u001B[0m     \u001B[1;31m# Download and prepare data\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1664\u001B[1;33m     builder_instance.download_and_prepare(\n\u001B[0m\u001B[0;32m   1665\u001B[0m         \u001B[0mdownload_config\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mdownload_config\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1666\u001B[0m         \u001B[0mdownload_mode\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mdownload_mode\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\datasets\\builder.py\u001B[0m in \u001B[0;36mdownload_and_prepare\u001B[1;34m(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, base_path, use_auth_token, **download_and_prepare_kwargs)\u001B[0m\n\u001B[0;32m    591\u001B[0m                             \u001B[0mlogger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mwarning\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m\"HF google storage unreachable. Downloading and preparing it from source\"\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    592\u001B[0m                     \u001B[1;32mif\u001B[0m \u001B[1;32mnot\u001B[0m \u001B[0mdownloaded_from_gcs\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 593\u001B[1;33m                         self._download_and_prepare(\n\u001B[0m\u001B[0;32m    594\u001B[0m                             \u001B[0mdl_manager\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mdl_manager\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mverify_infos\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mverify_infos\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mdownload_and_prepare_kwargs\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    595\u001B[0m                         )\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\datasets\\builder.py\u001B[0m in \u001B[0;36m_download_and_prepare\u001B[1;34m(self, dl_manager, verify_infos, **prepare_split_kwargs)\u001B[0m\n\u001B[0;32m    679\u001B[0m             \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    680\u001B[0m                 \u001B[1;31m# Prepare split will record examples associated to the split\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 681\u001B[1;33m                 \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_prepare_split\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0msplit_generator\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mprepare_split_kwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    682\u001B[0m             \u001B[1;32mexcept\u001B[0m \u001B[0mOSError\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    683\u001B[0m                 raise OSError(\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\datasets\\builder.py\u001B[0m in \u001B[0;36m_prepare_split\u001B[1;34m(self, split_generator)\u001B[0m\n\u001B[0;32m   1131\u001B[0m         \u001B[0mgenerator\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_generate_tables\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m**\u001B[0m\u001B[0msplit_generator\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mgen_kwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1132\u001B[0m         \u001B[1;32mwith\u001B[0m \u001B[0mArrowWriter\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mfeatures\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0minfo\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfeatures\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mpath\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mfpath\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0mwriter\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1133\u001B[1;33m             for key, table in utils.tqdm(\n\u001B[0m\u001B[0;32m   1134\u001B[0m                 \u001B[0mgenerator\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0munit\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;34m\" tables\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mleave\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mFalse\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdisable\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mTrue\u001B[0m  \u001B[1;31m# bool(logging.get_verbosity() == logging.NOTSET)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1135\u001B[0m             ):\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\tqdm\\notebook.py\u001B[0m in \u001B[0;36m__iter__\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    255\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m__iter__\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    256\u001B[0m         \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 257\u001B[1;33m             \u001B[1;32mfor\u001B[0m \u001B[0mobj\u001B[0m \u001B[1;32min\u001B[0m \u001B[0msuper\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtqdm_notebook\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m__iter__\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    258\u001B[0m                 \u001B[1;31m# return super(tqdm...) will not catch exception\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    259\u001B[0m                 \u001B[1;32myield\u001B[0m \u001B[0mobj\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\tqdm\\std.py\u001B[0m in \u001B[0;36m__iter__\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m   1166\u001B[0m         \u001B[1;31m# (note: keep this check outside the loop for performance)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1167\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mdisable\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1168\u001B[1;33m             \u001B[1;32mfor\u001B[0m \u001B[0mobj\u001B[0m \u001B[1;32min\u001B[0m \u001B[0miterable\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1169\u001B[0m                 \u001B[1;32myield\u001B[0m \u001B[0mobj\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1170\u001B[0m             \u001B[1;32mreturn\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\Anaconda3\\envs\\book\\lib\\site-packages\\datasets\\packaged_modules\\csv\\csv.py\u001B[0m in \u001B[0;36m_generate_tables\u001B[1;34m(self, files)\u001B[0m\n\u001B[0;32m    177\u001B[0m                     \u001B[1;32myield\u001B[0m \u001B[1;33m(\u001B[0m\u001B[0mfile_idx\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mbatch_idx\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mpa_table\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    178\u001B[0m             \u001B[1;32mexcept\u001B[0m \u001B[0mValueError\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 179\u001B[1;33m                 \u001B[0mlogger\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0merror\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34mf\"Failed to read file '{csv_file_reader.f}' with error {type(e)}: {e}\"\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    180\u001B[0m                 \u001B[1;32mraise\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mAttributeError\u001B[0m: 'TextFileReader' object has no attribute 'f'"
     ]
    }
   ],
   "source": [
    "#读取自定义数据集\n",
    "from datasets import load_dataset\n",
    "land = load_dataset(\"csv\", data_files = {\n",
    "    'train': ['train.csv'],\n",
    "    'validation' : ['valid.csv']\n",
    "})\n",
    "land[\"train\"][0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}